Day 1: Sequence analysis using TraMineR - EDA

Day 2: Sequence analysis using TraMineR - Matching, clustering

Day 3: Association rule mining using priori algorithm

Loading libaries

library(readr)
library(ggplot2)
library(data.table)
library(TraMineR)
library(dplyr)
library(tidyr)
library(TraMineR)
data(mvad)
seqstatl(mvad[, 17:86])
[1] "employment"  "FE"          "HE"          "joblessness" "school"      "training"   
mvad.alphabet <- c("employment", "FE", "HE", "joblessness", "school", 
    "training")
mvad.labels <- c("employment", "further education", "higher education", 
    "joblessness", "school", "training")
mvad.scodes <- c("EM", "FE", "HE", "JL", "SC", "TR")
mvad.seq <- seqdef(mvad, 17:86, alphabet = mvad.alphabet, states = mvad.scodes, 
    labels = mvad.labels, xtstep = 6)
 [>] state coding:
       [alphabet]  [label]  [long label] 
     1  employment  EM       employment
     2  FE          FE       further education
     3  HE          HE       higher education
     4  joblessness JL       joblessness
     5  school      SC       school
     6  training    TR       training
 [>] 712 sequences in the data set
 [>] min/max sequence length: 70/70
seqiplot(mvad.seq, with.legend = FALSE, title= "Index plot (10 first sequences")
 [!!] In seqiplot() : title is deprecated, use main instead.

seqfplot(mvad.seq, withlegend = F, title = "Sequence frequency plot bar width proportional to the frequencies")
 [!!] In seqfplot() : title is deprecated, use main instead.
 [!!] In seqfplot() : withlegend is deprecated, use with.legend instead.

 seqdplot(mvad.seq, withlegend = F,  title = "State distribution plot")
 [!!] In seqdplot() : title is deprecated, use main instead.
 [!!] In seqdplot() : withlegend is deprecated, use with.legend instead.

Import log data

lasi21_logdata <- read_csv("lasi21_logdata.csv")

── Column specification ─────────────────────────────────────────────────────────────────────
cols(
  id = col_double(),
  sas_id_site = col_double(),
  site_type = col_character(),
  date_time = col_datetime(format = ""),
  spent_time = col_double(),
  action = col_character(),
  instancename = col_character(),
  `avg score` = col_double(),
  PassFlag = col_double()
)
lasi21_logdata$site_type <- ifelse(grepl("ssignment", lasi21_logdata$instancename), "assignment",lasi21_logdata$site_type )
lasi21_logdata$site_type <- ifelse(grepl("exam", lasi21_logdata$instancename), "assignment",lasi21_logdata$site_type )

lasi21_logdata <- as.data.table(lasi21_logdata[,c("id","date_time","spent_time","site_type","instancename","PassFlag","avg score")])
head(lasi21_logdata)

Data pre-processing

# convert ms to minutes
lasi21_logdata$spent_time_m <- round(lasi21_logdata$spent_time/60000, digits=1)

# filter out all spent_time < 6s
lasi21_logdata2 <- lasi21_logdata %>% filter(spent_time>6000)

# create a flag for session break (the first click per student)
lasi21_logdata2$session_flag = 0
lasi21_logdata2 <- lasi21_logdata2 %>% arrange(id,date_time,spent_time) %>% 
    group_by(id) %>%
    mutate(session_flag  = +(row_number() %in% 1))

# create a flag for session break (aka where time spent > 30 minutes)
lasi21_logdata2$session_flag <- ifelse(lasi21_logdata2$spent_time_m>30, 1, lasi21_logdata2$session_flag)

# create session number 
lasi21_logdata2$session_num = cumsum(lasi21_logdata2$session_flag)

# remove session break
lasi21_logdata2 <- as.data.table(lasi21_logdata2)
lasi21_logdata2 <- lasi21_logdata2[session_flag!=1,]


# for each learning session, calculate the cummulative time spent
lasi21_logdata2 <- lasi21_logdata2 %>% 
                        arrange(id,date_time,spent_time) %>% 
                        group_by(session_num) %>% 
                        mutate(spent_time_m_cum = cumsum(spent_time_m))

# create time unit as 1/10 of a minute (6s)
lasi21_logdata2$time_unit <- round(lasi21_logdata2$spent_time_m_cum*10,digits=0)

lasi21_logdata_session <- lasi21_logdata2 %>% group_by(session_num) %>% top_n(1, spent_time_m_cum)

hist(lasi21_logdata_session$spent_time_m_cum, main="Learning session length", xlab = "Minutes", breaks=200)


# Remove learning session < 5 mins
lasi21_logdata2 <- merge(lasi21_logdata2,lasi21_logdata_session[lasi21_logdata_session$spent_time_m_cum>5 & lasi21_logdata_session$spent_time_m_cum<120, c("session_num")])

Transform data into STS format

# create a function to expand time unit for each learning session

f1 <- function(x1){
    x1 <- 1:max(x1)
    m1 <- max(c(length(x1)))
    length(x1) <- m1
    list(time_unit = x1)
}

# create a subset
lasi21_logdata3 <- lasi21_logdata2[,c("session_num","time_unit","site_type")]

# create an expanded df
test <- setDT(lasi21_logdata3)[, f1(time_unit), .(session_num)]

# merge with log data
test <- merge(test,lasi21_logdata3,by=c("session_num","time_unit"),all.x=TRUE)

# fill missing upward
test <- test %>% fill(site_type,.direction = "up")
# Sequence length distribution
hist(lasi21_logdata3[,max(time_unit), by="session_num"]$V1, breaks=100, main="Length of learning session", xlab='Sequence length')

# cummulative plot of seq length
plot(ecdf(lasi21_logdata3[,max(time_unit), by="session_num"]$V1), main ='Cummulative distribution of seq length')

Define sequences

# define sequences columns
log_sts.seq <- seqdef(log_sts,2:1200)
 [>] found missing values ('NA') in sequence data
 [>] preparing 18602 sequences
 [>] coding void elements with '%' and missing values with '*'
 [>] 12 distinct states appear in the data: 
     1 = assignment
     2 = collaborate
     3 = content
     4 = forumng
     5 = glossary
     6 = homepage
     7 = questionnaire
     8 = resource
     9 = studio
     10 = subpage
     11 = url
     12 = wiki
 [>] state coding:
       [alphabet]    [label]       [long label] 
     1  assignment    assignment    assignment
     2  collaborate   collaborate   collaborate
     3  content       content       content
     4  forumng       forumng       forumng
     5  glossary      glossary      glossary
     6  homepage      homepage      homepage
     7  questionnaire questionnaire questionnaire
     8  resource      resource      resource
     9  studio        studio        studio
     10  subpage       subpage       subpage
     11  url           url           url
     12  wiki          wiki          wiki
 [>] 18602 sequences in the data set
 [>] min/max sequence length: 51/1199

EDA

# plot the first 10 sequences with length=100
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqiplot(log_sts.seq[611:631,1:100], with.legend = F, main = "Index plot (10 first sequences)")
seqlegend(log_sts.seq)

# State distribution plot
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqdplot(log_sts.seq[,1:200], main = "State distribution plot", with.legend = FALSE)
seqlegend(log_sts.seq)

# Sequence frequency plot
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqfplot(log_sts.seq[,1:200], main = "Sequence frequency plot", with.legend = FALSE, pbarw = TRUE)
seqlegend(log_sts.seq)

# mean time by state
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqmtplot(log_sts.seq[,1:200], main = "Mean time plot", with.legend = FALSE)
seqlegend(log_sts.seq)

# Stability within sequences
# Shannon's entropy as a measure of the diversity of states observed at the considered time point
# It equals 0 when all cases are in the same state  (it is thus easy to predict in which state an individual is)
# It is maximum when the cases are equally distributed between the states
seqHtplot(log_sts.seq[,1:200], title = "Entropy index")
 [!!] In seqHtplot() : title is deprecated, use main instead.

# Turbulence
Turbulence <- seqST(log_sts.seq[,1:200])
summary(Turbulence)
   Turbulence    
 Min.   : 1.000  
 1st Qu.: 2.000  
 Median : 4.038  
 Mean   : 4.704  
 3rd Qu.: 6.641  
 Max.   :32.312  
hist(Turbulence, col = "cyan", main = "Sequence turbulence",breaks=50)

# Transitions of events

# define seq transitions
log_sts.seqe <- seqecreate(log_sts.seq[,1:200])

# find frequent subsequences
fsubseq <- seqefsub(log_sts.seqe, pMinSupport = 0.05)
 [!!] In seqefsub() : pMinSupport is deprecated, use pmin.support instead.
# plot 15 most frequent subsquences
plot(fsubseq[1:15], col = "cyan", main="Top 15 frequent subsequences")

Sequence similarities and clustering

# Compute sequence distances using OM with transition rate as substitution cost
log_sts.seq.om1 <- round(seqdist(log_sts.seq[1:1000,1:200], method = "OM", indel = 1, sm = "CONSTANT"),2)
 [>] 1000 sequences with 12 distinct states
 [>] Computing sm with seqcost using  CONSTANT
 [>] creating 12x12 substitution-cost matrix using 2 as constant value
 [>] 846 distinct  sequences 
 [>] min/max sequence lengths: 51/200
 [>] computing distances using the OM metric
 [>] elapsed time: 28.993 secs

fviz_nbclust(log_sts.seq.om1, FUN = hcut, method = "silhouette")+
labs(subtitle = "Silhouette method")

cluster3 <- cutree(clusterward, k = 3)
cluster3 <- factor(HC3c, labels = paste("Type", 1:3))
table(cluster3)
cluster3
Type 1 Type 2 Type 3 
   442    291    267 
# frequency plot of sequences by cluster
seqfplot(log_sts.seq[1:1000,1:200], group = cluster3, pbarw = T)

seqmtplot(log_sts.seq[1:1000,1:200], group = cluster3)

# layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqdplot(log_sts.seq[1:1000,1:200], group = cluster3, border = NA, with.legend = T)

# seqlegend(log_sts.seq)
covar$grade <- rnorm(1000, mean=70, sd=10); covar$grade <- covar$grade[covar$grade > 1 & covar$grade < 100]
Error in `$<-.data.frame`(`*tmp*`, grade, value = c(72.1121413037559,  : 
  replacement has 998 rows, data has 1000
reglog <- list()
for (i in 1:length(levels(cluster3))) {
reglog[[i]] <- glm((cluster3 == levels(cluster3)[i]) ~ condition + grade + surveymetric1 + surveymetric2, family = "binomial", data = covar)}
tbls <- list()
for (i in 1:length(levels(cluster3))) {tbls[[i]] <- tbl_regression(reglog[[i]], exponentiate = TRUE)}

tbl_merge(
    tbls = tbls,
    tab_spanner = c("**Type 1**", "**Type 2**", "**Type 3**")
  )
Characteristic Type 1 Type 2 Type 3
OR1 95% CI1 p-value OR1 95% CI1 p-value OR1 95% CI1 p-value
condition
Control — — — — — —
Treatment 0.90 0.70, 1.16 0.4 1.15 0.87, 1.51 0.3 0.99 0.75, 1.31 >0.9
grade 1.00 0.99, 1.02 0.5 0.99 0.98, 1.01 0.4 1.00 0.99, 1.02 0.8
surveymetric1 0.99 0.71, 1.36 >0.9 0.87 0.61, 1.24 0.4 1.18 0.82, 1.69 0.4
surveymetric2 0.98 0.80, 1.19 0.8 0.99 0.80, 1.23 >0.9 1.04 0.83, 1.30 0.7

1 OR = Odds Ratio, CI = Confidence Interval

Comparison of sequences

LS0tCnRpdGxlOiAiVGVtcG9yYWwgYW5kIHNlcXVlbnRpYWwgYW5hbHlzaXMgZm9yIGxlYXJuaW5nIGFuYWx5dGljcyIKYXV0aG9yOiAiUXVhbiBOZ3V5ZW4sIFBvc3Rkb2N0b3JhbCBGZWxsb3csIFNjaG9vbCBvZiBJbmZvcm1hdGlvbiwgVW5pdmVyc2l0eSBvZiBNaWNoaWdhbiIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKCiMgRGF5IDE6IFNlcXVlbmNlIGFuYWx5c2lzIHVzaW5nIFRyYU1pbmVSIC0gRURBCiMgRGF5IDI6IFNlcXVlbmNlIGFuYWx5c2lzIHVzaW5nIFRyYU1pbmVSIC0gTWF0Y2hpbmcsIGNsdXN0ZXJpbmcKIyBEYXkgMzogQXNzb2NpYXRpb24gcnVsZSBtaW5pbmcgdXNpbmcgcHJpb3JpIGFsZ29yaXRobQoKIyMgTG9hZGluZyBsaWJhcmllcwoKYGBge3J9CmxpYnJhcnkocmVhZHIpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShkYXRhLnRhYmxlKQpsaWJyYXJ5KFRyYU1pbmVSKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KHRpZHlyKQpgYGAKCmBgYHtyfQpsaWJyYXJ5KFRyYU1pbmVSKQpkYXRhKG12YWQpCnNlcXN0YXRsKG12YWRbLCAxNzo4Nl0pCm12YWQuYWxwaGFiZXQgPC0gYygiZW1wbG95bWVudCIsICJGRSIsICJIRSIsICJqb2JsZXNzbmVzcyIsICJzY2hvb2wiLCAKICAgICJ0cmFpbmluZyIpCm12YWQubGFiZWxzIDwtIGMoImVtcGxveW1lbnQiLCAiZnVydGhlciBlZHVjYXRpb24iLCAiaGlnaGVyIGVkdWNhdGlvbiIsIAogICAgImpvYmxlc3NuZXNzIiwgInNjaG9vbCIsICJ0cmFpbmluZyIpCm12YWQuc2NvZGVzIDwtIGMoIkVNIiwgIkZFIiwgIkhFIiwgIkpMIiwgIlNDIiwgIlRSIikKbXZhZC5zZXEgPC0gc2VxZGVmKG12YWQsIDE3Ojg2LCBhbHBoYWJldCA9IG12YWQuYWxwaGFiZXQsIHN0YXRlcyA9IG12YWQuc2NvZGVzLCAKICAgIGxhYmVscyA9IG12YWQubGFiZWxzLCB4dHN0ZXAgPSA2KQoKYGBgCmBgYHtyfQpoZWFkKG12YWQpCmBgYAoKYGBge3J9CnNlcWlwbG90KG12YWQuc2VxLCB3aXRoLmxlZ2VuZCA9IEZBTFNFLCB0aXRsZT0gIkluZGV4IHBsb3QgKDEwIGZpcnN0IHNlcXVlbmNlcyIpCmBgYApgYGB7cn0Kc2VxSXBsb3QobXZhZC5zZXEsIHNvcnR2ID0gImZyb20uc3RhcnQiLCB3aXRoLmxlZ2VuZCA9IEZBTFNFKQpgYGAKYGBge3J9CnNlcWZwbG90KG12YWQuc2VxLCB3aXRobGVnZW5kID0gRiwgdGl0bGUgPSAiU2VxdWVuY2UgZnJlcXVlbmN5IHBsb3QgYmFyIHdpZHRoIHByb3BvcnRpb25hbCB0byB0aGUgZnJlcXVlbmNpZXMiKQpgYGAKYGBge3J9CiBzZXFkcGxvdChtdmFkLnNlcSwgd2l0aGxlZ2VuZCA9IEYsICB0aXRsZSA9ICJTdGF0ZSBkaXN0cmlidXRpb24gcGxvdCIpCmBgYAoKCiMgSW1wb3J0IGxvZyBkYXRhCgpgYGB7cn0KbGFzaTIxX2xvZ2RhdGEgPC0gcmVhZF9jc3YoImxhc2kyMV9sb2dkYXRhLmNzdiIpCmxhc2kyMV9sb2dkYXRhJHNpdGVfdHlwZSA8LSBpZmVsc2UoZ3JlcGwoInNzaWdubWVudCIsIGxhc2kyMV9sb2dkYXRhJGluc3RhbmNlbmFtZSksICJhc3NpZ25tZW50IixsYXNpMjFfbG9nZGF0YSRzaXRlX3R5cGUgKQpsYXNpMjFfbG9nZGF0YSRzaXRlX3R5cGUgPC0gaWZlbHNlKGdyZXBsKCJleGFtIiwgbGFzaTIxX2xvZ2RhdGEkaW5zdGFuY2VuYW1lKSwgImFzc2lnbm1lbnQiLGxhc2kyMV9sb2dkYXRhJHNpdGVfdHlwZSApCgpsYXNpMjFfbG9nZGF0YSA8LSBhcy5kYXRhLnRhYmxlKGxhc2kyMV9sb2dkYXRhWyxjKCJpZCIsImRhdGVfdGltZSIsInNwZW50X3RpbWUiLCJzaXRlX3R5cGUiLCJpbnN0YW5jZW5hbWUiLCJQYXNzRmxhZyIsImF2ZyBzY29yZSIpXSkKaGVhZChsYXNpMjFfbG9nZGF0YSkKYGBgCgojIERhdGEgcHJlLXByb2Nlc3NpbmcKYGBge3J9CiMgY29udmVydCBtcyB0byBtaW51dGVzCmxhc2kyMV9sb2dkYXRhJHNwZW50X3RpbWVfbSA8LSByb3VuZChsYXNpMjFfbG9nZGF0YSRzcGVudF90aW1lLzYwMDAwLCBkaWdpdHM9MSkKCiMgZmlsdGVyIG91dCBhbGwgc3BlbnRfdGltZSA8IDZzCmxhc2kyMV9sb2dkYXRhMiA8LSBsYXNpMjFfbG9nZGF0YSAlPiUgZmlsdGVyKHNwZW50X3RpbWU+NjAwMCkKCiMgY3JlYXRlIGEgZmxhZyBmb3Igc2Vzc2lvbiBicmVhayAodGhlIGZpcnN0IGNsaWNrIHBlciBzdHVkZW50KQpsYXNpMjFfbG9nZGF0YTIkc2Vzc2lvbl9mbGFnID0gMApsYXNpMjFfbG9nZGF0YTIgPC0gbGFzaTIxX2xvZ2RhdGEyICU+JSBhcnJhbmdlKGlkLGRhdGVfdGltZSxzcGVudF90aW1lKSAlPiUgCiAgICBncm91cF9ieShpZCkgJT4lCiAgICBtdXRhdGUoc2Vzc2lvbl9mbGFnICA9ICsocm93X251bWJlcigpICVpbiUgMSkpCgojIGNyZWF0ZSBhIGZsYWcgZm9yIHNlc3Npb24gYnJlYWsgKGFrYSB3aGVyZSB0aW1lIHNwZW50ID4gMzAgbWludXRlcykKbGFzaTIxX2xvZ2RhdGEyJHNlc3Npb25fZmxhZyA8LSBpZmVsc2UobGFzaTIxX2xvZ2RhdGEyJHNwZW50X3RpbWVfbT4zMCwgMSwgbGFzaTIxX2xvZ2RhdGEyJHNlc3Npb25fZmxhZykKCiMgY3JlYXRlIHNlc3Npb24gbnVtYmVyIApsYXNpMjFfbG9nZGF0YTIkc2Vzc2lvbl9udW0gPSBjdW1zdW0obGFzaTIxX2xvZ2RhdGEyJHNlc3Npb25fZmxhZykKCiMgcmVtb3ZlIHNlc3Npb24gYnJlYWsKbGFzaTIxX2xvZ2RhdGEyIDwtIGFzLmRhdGEudGFibGUobGFzaTIxX2xvZ2RhdGEyKQpsYXNpMjFfbG9nZGF0YTIgPC0gbGFzaTIxX2xvZ2RhdGEyW3Nlc3Npb25fZmxhZyE9MSxdCgoKIyBmb3IgZWFjaCBsZWFybmluZyBzZXNzaW9uLCBjYWxjdWxhdGUgdGhlIGN1bW11bGF0aXZlIHRpbWUgc3BlbnQKbGFzaTIxX2xvZ2RhdGEyIDwtIGxhc2kyMV9sb2dkYXRhMiAlPiUgCiAgICAgICAgICAgICAgICAgICAgICAgIGFycmFuZ2UoaWQsZGF0ZV90aW1lLHNwZW50X3RpbWUpICU+JSAKICAgICAgICAgICAgICAgICAgICAgICAgZ3JvdXBfYnkoc2Vzc2lvbl9udW0pICU+JSAKICAgICAgICAgICAgICAgICAgICAgICAgbXV0YXRlKHNwZW50X3RpbWVfbV9jdW0gPSBjdW1zdW0oc3BlbnRfdGltZV9tKSkKCiMgY3JlYXRlIHRpbWUgdW5pdCBhcyAxLzEwIG9mIGEgbWludXRlICg2cykKbGFzaTIxX2xvZ2RhdGEyJHRpbWVfdW5pdCA8LSByb3VuZChsYXNpMjFfbG9nZGF0YTIkc3BlbnRfdGltZV9tX2N1bSoxMCxkaWdpdHM9MCkKCmxhc2kyMV9sb2dkYXRhX3Nlc3Npb24gPC0gbGFzaTIxX2xvZ2RhdGEyICU+JSBncm91cF9ieShzZXNzaW9uX251bSkgJT4lIHRvcF9uKDEsIHNwZW50X3RpbWVfbV9jdW0pCgpoaXN0KGxhc2kyMV9sb2dkYXRhX3Nlc3Npb24kc3BlbnRfdGltZV9tX2N1bSwgbWFpbj0iTGVhcm5pbmcgc2Vzc2lvbiBsZW5ndGgiLCB4bGFiID0gIk1pbnV0ZXMiLCBicmVha3M9MjAwKQoKIyBSZW1vdmUgbGVhcm5pbmcgc2Vzc2lvbiA8IDUgbWlucwpsYXNpMjFfbG9nZGF0YTIgPC0gbWVyZ2UobGFzaTIxX2xvZ2RhdGEyLGxhc2kyMV9sb2dkYXRhX3Nlc3Npb25bbGFzaTIxX2xvZ2RhdGFfc2Vzc2lvbiRzcGVudF90aW1lX21fY3VtPjUgJiBsYXNpMjFfbG9nZGF0YV9zZXNzaW9uJHNwZW50X3RpbWVfbV9jdW08MTIwLCBjKCJzZXNzaW9uX251bSIpXSkKCmBgYAoKIyBUcmFuc2Zvcm0gZGF0YSBpbnRvIFNUUyBmb3JtYXQKYGBge3J9CiMgY3JlYXRlIGEgZnVuY3Rpb24gdG8gZXhwYW5kIHRpbWUgdW5pdCBmb3IgZWFjaCBsZWFybmluZyBzZXNzaW9uCgpmMSA8LSBmdW5jdGlvbih4MSl7CiAgICB4MSA8LSAxOm1heCh4MSkKICAgIG0xIDwtIG1heChjKGxlbmd0aCh4MSkpKQogICAgbGVuZ3RoKHgxKSA8LSBtMQogICAgbGlzdCh0aW1lX3VuaXQgPSB4MSkKfQoKIyBjcmVhdGUgYSBzdWJzZXQKbGFzaTIxX2xvZ2RhdGEzIDwtIGxhc2kyMV9sb2dkYXRhMlssYygic2Vzc2lvbl9udW0iLCJ0aW1lX3VuaXQiLCJzaXRlX3R5cGUiKV0KCiMgY3JlYXRlIGFuIGV4cGFuZGVkIGRmCnRlc3QgPC0gc2V0RFQobGFzaTIxX2xvZ2RhdGEzKVssIGYxKHRpbWVfdW5pdCksIC4oc2Vzc2lvbl9udW0pXQoKIyBtZXJnZSB3aXRoIGxvZyBkYXRhCnRlc3QgPC0gbWVyZ2UodGVzdCxsYXNpMjFfbG9nZGF0YTMsYnk9Yygic2Vzc2lvbl9udW0iLCJ0aW1lX3VuaXQiKSxhbGwueD1UUlVFKQoKIyBmaWxsIG1pc3NpbmcgdXB3YXJkCnRlc3QgPC0gdGVzdCAlPiUgZmlsbChzaXRlX3R5cGUsLmRpcmVjdGlvbiA9ICJ1cCIpCmBgYAoKCgpgYGB7cn0KIyBTZXF1ZW5jZSBsZW5ndGggZGlzdHJpYnV0aW9uCmhpc3QobGFzaTIxX2xvZ2RhdGEzWyxtYXgodGltZV91bml0KSwgYnk9InNlc3Npb25fbnVtIl0kVjEsIGJyZWFrcz0xMDAsIG1haW49Ikxlbmd0aCBvZiBsZWFybmluZyBzZXNzaW9uIiwgeGxhYj0nU2VxdWVuY2UgbGVuZ3RoJykKYGBgCgpgYGB7cn0KIyBjdW1tdWxhdGl2ZSBwbG90IG9mIHNlcSBsZW5ndGgKcGxvdChlY2RmKGxhc2kyMV9sb2dkYXRhM1ssbWF4KHRpbWVfdW5pdCksIGJ5PSJzZXNzaW9uX251bSJdJFYxKSwgbWFpbiA9J0N1bW11bGF0aXZlIGRpc3RyaWJ1dGlvbiBvZiBzZXEgbGVuZ3RoJykKYGBgCiMgRGVmaW5lIHNlcXVlbmNlcwoKYGBge3J9CiMgY29udmVydCB0byBTVFMgZm9ybWF0ICh3aWRlIGZvcm1hdCkKbG9nX3N0cyA8LSBzcHJlYWQodGVzdCwgdGltZV91bml0LCBzaXRlX3R5cGUpCgojIGRlZmluZSBzZXF1ZW5jZXMgY29sdW1ucwpsb2dfc3RzLnNlcSA8LSBzZXFkZWYobG9nX3N0cywyOjEyMDApCgpgYGAKCiMgRURBCgpgYGB7cn0KIyBwbG90IHRoZSBmaXJzdCAxMCBzZXF1ZW5jZXMgd2l0aCBsZW5ndGg9MTAwCmxheW91dChtYXRyaXgoYygxLDEsMSwyKSwgbnJvdz0xLCBieXJvdz1UUlVFKSkKc2VxaXBsb3QobG9nX3N0cy5zZXFbNjExOjYzMSwxOjEwMF0sIHdpdGgubGVnZW5kID0gRiwgbWFpbiA9ICJJbmRleCBwbG90ICgxMCBmaXJzdCBzZXF1ZW5jZXMpIikKc2VxbGVnZW5kKGxvZ19zdHMuc2VxKQpgYGAKYGBge3J9CiMgUGxvdCAyMDAgc2VxdWVuY2VzIHNvcnRlZCBieSBMQ1MgKGxvbmdlc3QgY29tbW9uIHN1YnNlcXVlbmNlKSBkaXN0YW5jZQpkaXN0Lm1vc3RmcmVxIDwtIHNlcWRpc3QobG9nX3N0cy5zZXEsIG1ldGhvZCA9ICJMQ1MiLCByZWZzZXEgPSAwKQpzZXFJcGxvdChsb2dfc3RzLnNlcVsxOjIwMCwxOjIwMF0sIGJvcmRlciA9IE5BLCBzb3J0diA9IGRpc3QubW9zdGZyZXEsIHdpdGgubGVnZW5kPUYpCmBgYAoKCmBgYHtyfQojIFN0YXRlIGRpc3RyaWJ1dGlvbiBwbG90CmxheW91dChtYXRyaXgoYygxLDEsMSwyKSwgbnJvdz0xLCBieXJvdz1UUlVFKSkKc2VxZHBsb3QobG9nX3N0cy5zZXFbLDE6MjAwXSwgbWFpbiA9ICJTdGF0ZSBkaXN0cmlidXRpb24gcGxvdCIsIHdpdGgubGVnZW5kID0gRkFMU0UpCnNlcWxlZ2VuZChsb2dfc3RzLnNlcSkKYGBgCgpgYGB7cn0KIyBTZXF1ZW5jZSBmcmVxdWVuY3kgcGxvdApsYXlvdXQobWF0cml4KGMoMSwxLDEsMiksIG5yb3c9MSwgYnlyb3c9VFJVRSkpCnNlcWZwbG90KGxvZ19zdHMuc2VxWywxOjIwMF0sIG1haW4gPSAiU2VxdWVuY2UgZnJlcXVlbmN5IHBsb3QiLCB3aXRoLmxlZ2VuZCA9IEZBTFNFLCBwYmFydyA9IFRSVUUpCnNlcWxlZ2VuZChsb2dfc3RzLnNlcSkKYGBgCmBgYHtyfQojIG1lYW4gdGltZSBieSBzdGF0ZQpsYXlvdXQobWF0cml4KGMoMSwxLDEsMiksIG5yb3c9MSwgYnlyb3c9VFJVRSkpCnNlcW10cGxvdChsb2dfc3RzLnNlcVssMToyMDBdLCBtYWluID0gIk1lYW4gdGltZSBwbG90Iiwgd2l0aC5sZWdlbmQgPSBGQUxTRSkKc2VxbGVnZW5kKGxvZ19zdHMuc2VxKQpgYGAKCmBgYHtyfQojIFN0YWJpbGl0eSB3aXRoaW4gc2VxdWVuY2VzCiMgU2hhbm5vbidzIGVudHJvcHkgYXMgYSBtZWFzdXJlIG9mIHRoZSBkaXZlcnNpdHkgb2Ygc3RhdGVzIG9ic2VydmVkIGF0IHRoZSBjb25zaWRlcmVkIHRpbWUgcG9pbnQKIyBJdCBlcXVhbHMgMCB3aGVuIGFsbCBjYXNlcyBhcmUgaW4gdGhlIHNhbWUgc3RhdGUgIChpdCBpcyB0aHVzIGVhc3kgdG8gcHJlZGljdCBpbiB3aGljaCBzdGF0ZSBhbiBpbmRpdmlkdWFsIGlzKQojIEl0IGlzIG1heGltdW0gd2hlbiB0aGUgY2FzZXMgYXJlIGVxdWFsbHkgZGlzdHJpYnV0ZWQgYmV0d2VlbiB0aGUgc3RhdGVzCnNlcUh0cGxvdChsb2dfc3RzLnNlcVssMToyMDBdLCB0aXRsZSA9ICJFbnRyb3B5IGluZGV4IikKYGBgCgpgYGB7cn0KIyBUdXJidWxlbmNlClR1cmJ1bGVuY2UgPC0gc2VxU1QobG9nX3N0cy5zZXFbLDE6MjAwXSkKc3VtbWFyeShUdXJidWxlbmNlKQpoaXN0KFR1cmJ1bGVuY2UsIGNvbCA9ICJjeWFuIiwgbWFpbiA9ICJTZXF1ZW5jZSB0dXJidWxlbmNlIixicmVha3M9NTApCgpgYGAKCmBgYHtyfQojIFRyYW5zaXRpb25zIG9mIGV2ZW50cwoKIyBkZWZpbmUgc2VxIHRyYW5zaXRpb25zCmxvZ19zdHMuc2VxZSA8LSBzZXFlY3JlYXRlKGxvZ19zdHMuc2VxWywxOjIwMF0pCgojIGZpbmQgZnJlcXVlbnQgc3Vic2VxdWVuY2VzCmZzdWJzZXEgPC0gc2VxZWZzdWIobG9nX3N0cy5zZXFlLCBwTWluU3VwcG9ydCA9IDAuMDUpCgojIHBsb3QgMTUgbW9zdCBmcmVxdWVudCBzdWJzcXVlbmNlcwpwbG90KGZzdWJzZXFbMToxNV0sIGNvbCA9ICJjeWFuIiwgbWFpbj0iVG9wIDE1IGZyZXF1ZW50IHN1YnNlcXVlbmNlcyIpCgpgYGAKCiMgU2VxdWVuY2Ugc2ltaWxhcml0aWVzIGFuZCBjbHVzdGVyaW5nCgpgYGB7cn0KIyBDb21wdXRlIHNlcXVlbmNlIGRpc3RhbmNlcyB1c2luZyBPTSB3aXRoIHRyYW5zaXRpb24gcmF0ZSBhcyBzdWJzdGl0dXRpb24gY29zdApsb2dfc3RzLnNlcS5vbTEgPC0gcm91bmQoc2VxZGlzdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBtZXRob2QgPSAiT00iLCBpbmRlbCA9IDEsIHNtID0gIkNPTlNUQU5UIiksMikKYGBgCmBgYHtyfQojIGFwcGx5IGFnZ2xvbWVyYXRlIGhpZXJhcmNoaWNhbCBjbHVzdGVyaW5nCmxpYnJhcnkoY2x1c3RlcikKY2x1c3RlcndhcmQgPC0gYWduZXMobG9nX3N0cy5zZXEub20xLCBkaXNzID0gVFJVRSwgbWV0aG9kID0gIndhcmQiKQpoY2QgPC0gYXMuZGVuZHJvZ3JhbShjbHVzdGVyd2FyZCkKcGxvdChoY2QsIHdoaWNoLnBsb3RzID0gMiwgbWFpbj0iRGVuZG9ncmFtIG9mIHNlcXVlbmNlcyBjbHVzdGVyaW5nIiwgbGVhZmxhYiA9ICJub25lIikKYGBgCmBgYHtyfQpsaWJyYXJ5KGZhY3RvZXh0cmEpCiMgUGxvdCBzY3JlZSBwbG90IHVzaW5nIHdzcwpmdml6X25iY2x1c3QobG9nX3N0cy5zZXEub20xLCBGVU4gPSBoY3V0LCBtZXRob2QgPSAid3NzIikKYGBgCmBgYHtyfQojIFBsb3QgU2lsaG91ZXR0ZSAKZnZpel9uYmNsdXN0KGxvZ19zdHMuc2VxLm9tMSwgRlVOID0gaGN1dCwgbWV0aG9kID0gInNpbGhvdWV0dGUiKSsKbGFicyhzdWJ0aXRsZSA9ICJTaWxob3VldHRlIG1ldGhvZCIpCmBgYApgYGB7cn0KIyBVc2Ugaz0zIGFzIHRoZSBudW1iZXIgb2YgY2x1c3RlcgpjbHVzdGVyMyA8LSBjdXRyZWUoY2x1c3RlcndhcmQsIGsgPSAzKQpjbHVzdGVyMyA8LSBmYWN0b3IoSEMzYywgbGFiZWxzID0gcGFzdGUoIlR5cGUiLCAxOjMpKQp0YWJsZShjbHVzdGVyMykKYGBgCmBgYHtyfQojIGZyZXF1ZW5jeSBwbG90IG9mIHNlcXVlbmNlcyBieSBjbHVzdGVyCnNlcWZwbG90KGxvZ19zdHMuc2VxWzE6MTAwMCwxOjIwMF0sIGdyb3VwID0gY2x1c3RlcjMsIHBiYXJ3ID0gVCkKYGBgCgpgYGB7cn0KIyBtZWFuIHRpbWUgcGxvdCBvZiBzdGF0ZXMgYnkgY2x1c3RlciBtZW1iZXJzaGlwCnNlcW10cGxvdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBncm91cCA9IGNsdXN0ZXIzKQpgYGAKYGBge3J9CiMgbGF5b3V0KG1hdHJpeChjKDEsMSwxLDIpLCBucm93PTEsIGJ5cm93PVRSVUUpKQpzZXFkcGxvdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBncm91cCA9IGNsdXN0ZXIzLCBib3JkZXIgPSBOQSwgd2l0aC5sZWdlbmQgPSBUKQojIHNlcWxlZ2VuZChsb2dfc3RzLnNlcSkKCmBgYAoKYGBge3J9CiMgbWFkZSB1cCBjb3ZhcmlhdGVzIGZvciAxMDAwIHNlcXVlbmNlcyAoZS5nLiwgY29uZGl0aW9uLCBncmFkZSwgc3VydmV5X21ldHJpYzEsIHN1cnZleV9tZXRyaWMyKQpjb3ZhciA8LSBkYXRhLmZyYW1lKG1hdHJpeChuY29sID0gNCwgbnJvdyA9IDEwMDApKQpjb2xuYW1lcyhjb3ZhcikgPC0gYygiY29uZGl0aW9uIiwgImdyYWRlIiwgInN1cnZleW1ldHJpYzEiLCAic3VydmV5bWV0cmljMiIpCgpzZXQuc2VlZCgxMjMpCmNvdmFyJGNvbmRpdGlvbiA8LSBzYW1wbGUoeD1jKCJDb250cm9sIiwiVHJlYXRtZW50IiksMTAwMCxwcm9iPWMoMC41LDAuNSkscmVwbGFjZT1UKQoKc2V0LnNlZWQoMTIzKQpjb3ZhciRncmFkZSA8LSByb3VuZChybm9ybSgxMDAwLCBtZWFuPTYwLCBzZD0xMCksMCkKCnNldC5zZWVkKDEyMykKY292YXIkc3VydmV5bWV0cmljMSA8LSBzYW1wbGUoeD0xOjUsIHByb2IgPSBjKDAuMSwwLjIsMC40LDAuMiwwLjEpLCByZXBsYWNlID0gVCkKCnNldC5zZWVkKDEyMykKY292YXIkc3VydmV5bWV0cmljMiA8LSBzYW1wbGUoeD0xOjUsIHByb2IgPSBjKDAuMSwwLjIsMC4zLDAuMywwLjEpLCByZXBsYWNlID0gVCkKYGBgCgpgYGB7cn0KbGlicmFyeShndHN1bW1hcnkpCgojIHJ1biBsb2dpc3RpYyByZWdyZXNzaW9uIGZvciBlYWNoIGNsdXN0ZXIgb2Ygc2VxdWVuY2VzCnJlZ2xvZyA8LSBsaXN0KCkKZm9yIChpIGluIDE6bGVuZ3RoKGxldmVscyhjbHVzdGVyMykpKSB7cmVnbG9nW1tpXV0gPC0gZ2xtKChjbHVzdGVyMyA9PSBsZXZlbHMoY2x1c3RlcjMpW2ldKSB+IAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb25kaXRpb24gKyBncmFkZSArIHN1cnZleW1ldHJpYzEgKyBzdXJ2ZXltZXRyaWMyLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZmFtaWx5ID0gImJpbm9taWFsIiwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBjb3Zhcil9CiMgY3JlYXRlIG5pY2Ugc3VtbWFyeSBvdXRwdXQgdXNpbmcgZ3RzdW1tYXJ5IHBhY2thZ2UgCiMgaHR0cDovL3d3dy5kYW5pZWxkc2pvYmVyZy5jb20vZ3RzdW1tYXJ5L2FydGljbGVzL3RibF9yZWdyZXNzaW9uLmh0bWwKdGJscyA8LSBsaXN0KCkKZm9yIChpIGluIDE6bGVuZ3RoKGxldmVscyhjbHVzdGVyMykpKSB7dGJsc1tbaV1dIDwtIHRibF9yZWdyZXNzaW9uKHJlZ2xvZ1tbaV1dLCBleHBvbmVudGlhdGUgPSBUUlVFKX0KCnRibF9tZXJnZSgKICAgIHRibHMgPSB0YmxzLAogICAgdGFiX3NwYW5uZXIgPSBjKCIqKlR5cGUgMSoqIiwgIioqVHlwZSAyKioiLCAiKipUeXBlIDMqKiIpCiAgKQpgYGAKCiMgQ29tcGFyaXNvbiBvZiBzZXF1ZW5jZXMKCmBgYHtyfQojIFBsb3Qgc2VxdWVuY2VzIGJ5IGdyb3VwCmRpc3QucmVmc2VxIDwtIHNlcWRpc3QobG9nX3N0cy5zZXFbMToxMDAwLDE6MjAwXSwgcmVmc2VxID0gMCwgbWV0aG9kID0gIkxDUyIpCgpzZXFJcGxvdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBncm91cCA9IGNvdmFyJGNvbmRpdGlvbiwgc29ydHYgPSBkaXN0LnJlZnNlcSwgd2l0aC5sZWdlbmQgPSAicmlnaHQiKQpgYGAKCgoK